%%capture
# Install the necessary packages
!pip install plotly pandas numpy matplotlib seaborn sentence-transformers sklearn

import plotly.graph_objs as go
import plotly.express as px
import pandas as pd
from sklearn.decomposition import PCA

# Example embeddings and titles
embeddings = [
    [0.8, -0.6, 0.3],
    [0.7, -0.5, 0.4],
    [0.8, -0.5, 0.3],
    [0.1, 0.9, -0.2],
    [-0.3, 0.4, 0.9],
    [0.0, 0.8, -0.3]
]

titles = [
    "The stock market is experiencing a significant downturn.",
    "The economy is showing signs of recovery.",
    "There is a major decline in the stock market today.",
    "The weather is sunny and pleasant.",
    "Apple releases its new iPhone model.",
    "It's raining heavily in the city."
]

# Convert vectors to a DataFrame
data = {
    'Title': titles,
    'X': [vec[0] for vec in embeddings],
    'Y': [vec[1] for vec in embeddings],
    'Z': [vec[2] for vec in embeddings]
}

df = pd.DataFrame(data)

# Create a 3D scatter plot
fig = px.scatter_3d(df, x='X', y='Y', z='Z', color='Title')

# Update marker size and remove text labels
fig.update_traces(marker=dict(size=5), selector=dict(mode='markers'))

# Set the layout of the plot
fig.update_layout(
    scene=dict(
        xaxis_title='PCA Component 1',
        yaxis_title='PCA Component 2',
        zaxis_title='PCA Component 3'
    ),
    title='3D Visualization of Financial News Titles w/ Manual Embeddings',
    legend_title='Titles',
    margin=dict(l=0, r=0, b=0, t=40)
)

fig.show()

from sentence_transformers import SentenceTransformer

# Example financial news titles
titles = [
    "The stock market is experiencing a significant downturn.",
    "The stock market saw a sharp decline today.",
    "Major losses hit the stock market.",
    "Tech stocks rally as market rebounds from losses.",
    "Technology shares surge following market recovery.",
    "Recovery drives tech stocks to new highs.",
    "Federal Reserve signals possible interest rate hike.",
    "Fed hints at increasing interest rates soon.",
    "Interest rate hike likely as suggested by the Federal Reserve.",
    "Bitcoin reaches new all-time high amid market optimism.",
    "Market optimism pushes Bitcoin to record high.",
    "Bitcoin hits unprecedented levels with market cheer.",
    "Oil prices surge due to supply chain disruptions.",
    "Supply chain issues cause a spike in oil prices.",
    "Oil prices increase sharply as supply chains falter."
]

# Load the SBERT model
model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')

# Compute embeddings
embeddings = model.encode(titles)

/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/sentence_transformers/cross_encoder/CrossEncoder.py:11: TqdmExperimentalWarning:

Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)

/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/huggingface_hub/file_download.py:1132: FutureWarning:

`resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.

import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity

# Calculate cosine similarity
similarity_matrix = cosine_similarity(embeddings)

# Mask the lower triangle and the diagonal of the similarity matrix
mask = np.triu(np.ones_like(similarity_matrix, dtype=bool), k=1)

# Plot the cosine similarity matrix as a half heatmap
plt.figure(figsize=(10, 8))
sns.heatmap(similarity_matrix, annot=True, mask=mask, cmap="Greens", fmt=".2f", 
            xticklabels=titles, yticklabels=titles, cbar_kws={'label': 'Cosine Similarity'})
plt.xticks(rotation=90, ha='center', fontsize=8)
plt.yticks(rotation=0, fontsize=8)
plt.title('Cosine Similarity of Financial News Titles')
plt.tight_layout()
plt.show()

import numpy as np
import plotly.graph_objects as go
from sklearn.decomposition import PCA

# Reduce dimensions (vector length from 384 to 3) to 3D for visualization using PCA
pca = PCA(n_components=3)  # PCA used for simplicity
reduced_embeddings = pca.fit_transform(embeddings)

# Create 3D scatter plot
fig = go.Figure()

for i, embedding in enumerate(reduced_embeddings):
    fig.add_trace(go.Scatter3d(
        x=[embedding[0]],
        y=[embedding[1]],
        z=[embedding[2]],
        mode='markers',  # Ensure markers are used
        marker=dict(size=5),  # Adjust the size as needed
        name=titles[i]  # Add the title to the legend
    ))

# Update layout
fig.update_layout(
    scene=dict(
        xaxis_title='PCA Component 1',
        yaxis_title='PCA Component 2',
        zaxis_title='PCA Component 3'
    ),
    title='3D Visualization of Financial News Titles Using SBERT',
    legend_title_text='Titles',
    margin=dict(l=0, r=0, b=0, t=40)
)

fig.show()

import os
import pickle
import torch
from sentence_transformers import SentenceTransformer
from typing import List, Dict

class EmbeddingsProcessor:
    def __init__(self, model: SentenceTransformer, cache_path: str, device: str = None):
        """
        Initializes the EmbeddingsProcessor with a SentenceTransformer model,
        a path for caching embeddings, and an optional device specification.

        Parameters:
        model (SentenceTransformer): The SentenceTransformer model used for generating embeddings.
        cache_path (str): Path to store the cache of embeddings.
        device (str): Preferred device for computation ('cuda', 'mps', 'cpu', or None).
                      If None, the best available device will be used.
        """
        self.model = model
        self.cache_path = cache_path
        self.device = self._get_device(device)  # Determine the best available device
        self.embeddings_cache = self._load_embeddings_cache()  # Load cached embeddings if available

    def _get_device(self, preferred_device: str = None) -> torch.device:
        """
        Determines the best available device for computation.

        Parameters:
        preferred_device (str): The preferred device (optional).

        Returns:
        torch.device: The best available device.
        """
        if preferred_device:
            # Attempt to use the preferred device
            device = torch.device(preferred_device)
            if device.type == 'cuda' and torch.cuda.is_available():
                return device
            elif device.type == 'mps' and torch.backends.mps.is_available():
                return device
            elif device.type == 'rocm' and torch.cuda.is_available():
                return device  # ROCm is detected as CUDA
        # Default to the best available device
        if torch.cuda.is_available():
            return torch.device('cuda')
        elif torch.backends.mps.is_available():
            return torch.device('mps')
        else:
            return torch.device('cpu')

    def _load_embeddings_cache(self) -> Dict[str, torch.Tensor]:
        """
        Loads embeddings cache from the specified file path.

        Returns:
        dict: The loaded embeddings cache or an empty dictionary if the cache does not exist.
        """
        if os.path.exists(self.cache_path):
            # Load the cache if the file exists
            with open(self.cache_path, 'rb') as f:
                return pickle.load(f)
        return {}  # Return an empty dictionary if the cache file does not exist

    def _save_embeddings_cache(self):
        """
        Saves the current embeddings cache to the specified file path.
        """
        with open(self.cache_path, 'wb') as f:
            pickle.dump(self.embeddings_cache, f)

    def _compute_embeddings(self, corpus: List[str]) -> torch.Tensor:
        """
        Computes and caches embeddings for a given corpus of texts.

        Parameters:
        corpus (list): A list of texts to compute embeddings for.

        Returns:
        torch.Tensor: A tensor of normalized embeddings.
        """
        embeddings = []
        new_embeddings = {}
        for text in corpus:
            if text not in self.embeddings_cache:
                # Compute embedding if it's not already cached
                embedding = self.model.encode(text, convert_to_tensor=True)
                new_embeddings[text] = embedding.cpu()  # Store the embedding on CPU to save GPU memory
            else:
                new_embeddings[text] = self.embeddings_cache[text]
            embeddings.append(new_embeddings[text])

        if new_embeddings:
            # Update the cache if new embeddings were computed
            self.embeddings_cache.update(new_embeddings)
            self._save_embeddings_cache()

        # Stack embeddings into a single tensor
        embeddings_tensor = torch.stack(embeddings)
        embeddings_tensor = embeddings_tensor.to(self.device)  # Move the tensor to the desired device
        # Normalize the embeddings by their norms
        norms = embeddings_tensor.norm(p=2, dim=1, keepdim=True)
        normalized_embeddings = embeddings_tensor.div(norms)
        return normalized_embeddings

    def process_batch(self, corpus: List[str]) -> torch.Tensor:
        """
        Processes a batch of texts and returns their embeddings.

        Parameters:
        corpus (list): A list of texts to process.

        Returns:
        torch.Tensor: A tensor of embeddings for the given texts.
        """
        embeddings = self._compute_embeddings(corpus)
        return embeddings

# Example usage of the EmbeddingsProcessor
# Load a SentenceTransformer model
model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
# Path to store the embeddings cache
cache_path = 'embeddings_cache.pkl'
# Initialize the processor
processor = EmbeddingsProcessor(model, cache_path)

# Example corpus of texts
corpus = titles # Using the financial news titles from the previous example
# Process the corpus to get embeddings
embeddings = processor.process_batch(corpus)
print(corpus[0], embeddings[0][0:10]) # you can see here that for the first title, SBERT has generated a 384-dimensional embedding (only 10 dimensions are shown here to avoid a long output)

/opt/hostedtoolcache/Python/3.12.3/x64/lib/python3.12/site-packages/huggingface_hub/file_download.py:1132: FutureWarning:

`resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.

The stock market is experiencing a significant downturn. tensor([ 0.0315, -0.0482,  0.0491,  0.0356, -0.0831,  0.0144, -0.0546,  0.0748,
         0.0136, -0.0191])

Financial News Strategy | 2. Sentence Embeddings¶

Open article on Medium | Colab | Github ¶

Background¶

Introduction¶

Model of Choice: SBERT - all-MiniLM-L6-v2 ¶

High Quality¶

Efficiency¶

Versatility and Ease of Use¶

Pre-Trained on Diverse Data¶

Accuracy in Semantic Matching¶

Sentence Embeddings using SBERT¶

A few insights based on the results:¶

Considerations in Interpretation¶

Optimized Embedding Processor with GPU Support for Sentence Transformers¶

Financial News Strategy | 2. Sentence Embeddings¶

Open article on Medium | Colab | Github¶

Related Articles¶

Background¶

Introduction¶

Model of Choice: SBERT - all-MiniLM-L6-v2¶

High Quality¶

Efficiency¶

Versatility and Ease of Use¶

Pre-Trained on Diverse Data¶

Accuracy in Semantic Matching¶

Sentence Embeddings using SBERT¶

A few insights based on the results:¶

Considerations in Interpretation¶

Related Articles¶

Optimized Embedding Processor with GPU Support for Sentence Transformers¶

Open article on Medium | Colab | Github ¶

Model of Choice: SBERT - all-MiniLM-L6-v2 ¶